package main

import (
	//"bufio"
	"bytes"
	"code.google.com/p/go.text/encoding/simplifiedchinese"
	"code.google.com/p/go.text/transform"
	"flag"
	"fmt"
	goq "github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	//"runtime"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	main_domain = "" //主域名
	waitgroup   = &sync.WaitGroup{}
	LOG         = ""
	ErrLOG      = ""
)

func main() {
	var citycode = flag.String("citycode", "qingdao", "请输入城市编号") //城市编号
	flag.Parse()
	main_domain = "http://" + *citycode + ".qfang.com"
	LOG = *citycode + ".qfang.com.out.log"
	ErrLOG = *citycode + ".qfang.com.err.log"

	tmpurl := fmt.Sprintf("%s/garden", main_domain)
	doc, e := getDoc(tmpurl)
	if e != nil {
		fmt.Println(e.Error())
		os.Exit(-1)
	}

	tmpRecordCount := filterStr(doc.Find(".list_header .color_f60").Text())
	RecordCount, e := strconv.Atoi(tmpRecordCount)
	if e != nil {
		os.Exit(-1)
	}
	PageSize := 20
	pageCount := RecordCount / PageSize
	if RecordCount%PageSize > 0 {
		pageCount++
	}

	for i := pageCount; i >= 1; i -= 5 {
		for j := i; j > i-5 && j >= 1; j-- {
			getCommunityList(i)
		}
	}
	fmt.Println("done.")
}

//获取楼盘列表
func getCommunityList(curpage int) {
	defer waitgroup.Done()
	tmpurl := fmt.Sprintf("%s/garden/n%d/", main_domain, curpage)
	doc, e := getDoc(tmpurl)
	if e != nil {
		s(tmpurl + e.Error())
		return
	}
	doc.Find(".listing_cont").Children().Each(func(i int, sec *goq.Selection) {
		var (
			communityname    string //楼盘名称
			district, block  string //区域、板块
			salenum, rentnum string //出售、出租条数
			avgprice         string //均价
			link             string //链接
		)

		link, ok := sec.Find(".gardenInfo h3 a").Attr("href")
		if !ok {
			return
		}

		communityname = filterStr(sec.Find(".gardenInfo h3").Text())
		if "" == communityname {
			communityname = "--"
		}
		houseNum := sec.Find("p:contains('套')")
		switch houseNum.Find("a").Length() {
		case 2:
			{
				salenum = filterStr(houseNum.Find("a").Eq(0).Text())
				rentnum = filterStr(houseNum.Find("a").Eq(1).Text())
			}
		case 1:
			{
				if sec.Find("p:contains('二手房：')").Length() > 0 {
					salenum = filterStr(houseNum.Find("a").Eq(0).Text())
					rentnum = "--"
				} else {
					rentnum = filterStr(houseNum.Find("a").Eq(0).Text())
					salenum = "--"
				}
			}
		default:
			{
				salenum = "--"
				rentnum = "--"
			}
		}

		tmpa := strings.Replace(sec.Find("p:contains('查看地图')").Text(), "[查看地图]", "", -1)
		tmpa = filterStr(tmpa)
		reg := regexp.MustCompile(`\t+`)
		tmpa = reg.ReplaceAllString(tmpa, "\t")

		tmparr := strings.Split(tmpa, "\t")
		if len(tmparr) == 3 {
			district = tmparr[0]
			block = tmparr[1]
		} else {
			district = "--"
			block = "--"
		}
		avgprice = filterStr(sec.Find("p .price").Text())
		if "" == avgprice {
			avgprice = "--"
		}

		agentlist := getAgentList(link)
		agentnum := len(agentlist)
		for k, _ := range agentlist {
			tmpstr := fmt.Sprintf("%s||%s||%s||%s||%s||%s||%d||%s", communityname, district, block, salenum, rentnum, avgprice, agentnum, k)
			outlog(tmpstr)
		}
	})
}

//获取在此楼盘发布房源的人数
func getAgentList(url string) map[string]bool {
	agentlist := make(map[string]bool)
	tmpurl := fmt.Sprintf("%s%s", main_domain, url)
	//出售经纪人
	tmpurlesf := strings.Replace(tmpurl, "desc", "sale", 1)
	setAgentList(tmpurlesf, agentlist)

	//出租经纪人
	tmpurlzf := strings.Replace(tmpurl, "desc", "rent", 1)
	setAgentList(tmpurlzf, agentlist)

	return agentlist
}

//获取经纪人列表
func setAgentList(url string, agentlist map[string]bool) {
	doc, e := getDoc(url)
	if e != nil {
		return
	}
	PageSize := 20
	RecordCount, e := strconv.Atoi(doc.Find(".list_header .color_f60").Text())

	if e != nil {
		RecordCount = 0
	}

	pageCount := RecordCount / PageSize
	if RecordCount%PageSize > 0 {
		pageCount++
	}

	for i := 1; i <= pageCount; i++ {
		pageurl := fmt.Sprintf("%s/f%d/", url, i)
		doc, e := getDoc(pageurl)
		if e != nil {
			continue
		}
		doc.Find(".listing_cont").Children().Each(func(i int, sec *goq.Selection) {
			link, ok := sec.Find(".NewStatus a").Attr("href")
			if ok {
				tmplink := fmt.Sprintf("%s%s", main_domain, link)
				doc2, e := getDoc(tmplink)
				if e == nil {
					agentname := filterStr(doc2.Find(".broker_basic_name").Text())
					mobile := strings.Replace(filterStr(doc2.Find(".mtel_num").Text()), " ", "", -1)
					if agentname == "" || mobile == "" {
						return
					}
					agentinfo := agentname + "||" + mobile
					if !agentlist[agentinfo] {
						agentlist[agentinfo] = true
					}
				}
			}
		})
	}
}

//获取doc
func getDoc(url string) (*goq.Document, error) {
	res, e := getResp(url)
	if e != nil {
		return nil, e
	}
	doc, e := goq.NewDocumentFromResponse(res)
	if e != nil {
		return nil, e
	}
	return doc, nil
}

//获取response
func getResp(url string) (*http.Response, error) {
	req, e := http.NewRequest("GET", url, nil)
	req.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	//req.Header.Add("Accept-Encoding", "gzip,deflate,sdch")
	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.8")
	req.Header.Add("Cache-Control", "max-age=0")
	req.Header.Add("Proxy-Connection", "keep-alive")
	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")
	c := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(25 * time.Second)
				c, err := net.DialTimeout(netw, addr, time.Second*20)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	var resp *http.Response
	//尝试三次
	for retry := 0; retry < 3; retry++ {
		resp, e = c.Do(req)
		if e != nil {
			time.Sleep(time.Second * 1)
		} else {
			break
		}
	}
	if e != nil {
		return nil, e
	}
	return resp, nil
}

//gbk->utf8
func G2U(s string) string {
	ret, e := gbktoutf8([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

//utf8->gbk
func U2G(s string) string {
	ret, e := utf8togbk([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

func gbktoutf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}
func utf8togbk(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewEncoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func filterStr(str string) string {
	str = strings.Replace(str, "\r\n", "", -1)
	str = strings.Replace(str, "\r", "", -1)
	str = strings.Replace(str, "\n", "", -1)
	str = strings.Replace(str, "|", "", -1)
	str = strings.TrimSpace(str)
	return str
}

func s(a ...interface{}) {
	fmt.Println(a)
}

//outlog
func outlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_log.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}

//errlog
func errlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_err.txt", pwd, ErrLOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}
